print "Get the AOL query term freq first."
fileNameList = []
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-01.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-02.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-03.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-04.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-05.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-06.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-07.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-08.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-09.txt")
fileNameList.append("/data5/team/weijiang/AOLQueryTrace/user-ct-test-collection-10.txt")
# print len(fileNameList)

outputFileHandler = open("/data5/team/weijiang/the_new_trip_of_feature_generation/final-queryTermsFreqFromAOL.txt", "w")

queryTermFreqDict = {}
queryTermList = []

for index,name in enumerate(fileNameList): 
    print "Processing ",name
    inputAuxFileHandler = open(name, "r")
    inputAuxFileHandler.readline()
    
    for index2,line in enumerate( inputAuxFileHandler.readlines() ):
        query = line.strip().split("\t")[1].strip()
        
        # This is the filter for filtering out the chars which don't index.
        for i in range(0,len(query)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(query[i]) >= 48 and ord(query[i]) < 58) or (ord(query[i]) >= 65 and ord(query[i]) < 91) or (ord(query[i]) >= 97 and ord(query[i]) < 123) or (ord(query[i]) == 32) ):
                # Just replace them with a space.
                query = query[:i] + " " + query[i+1:]
        
        if query.strip() == "":
            pass
        else:
            for queryTerm in query.split(" "):
                if queryTerm not in queryTermFreqDict and queryTerm != "":
                    queryTermFreqDict[queryTerm] = 1
                elif queryTerm in queryTermFreqDict and queryTerm != "":
                    queryTermFreqDict[queryTerm] += 1
        
        # for debug purpose.
        # if index2 == 100:
        #    break
   
# print "queryTermFreqDict:",queryTermFreqDict
queryTermList = queryTermFreqDict.keys()
queryTermList.sort(cmp=None, key=None, reverse=False)
for queryTerm in queryTermList:
    outputFileHandler.write(queryTerm + " " + str(queryTermFreqDict[queryTerm]) + "\n")  
        
outputFileHandler.close()
print "Done"


